{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "![title](img/dilated_cnn.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "pip3 install tensor2tensor\n",
    "\"\"\"\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensor2tensor.utils import beam_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'batch_size': 128,\n",
    "    'text_iter_step': 25,\n",
    "    'seq_len': 200,\n",
    "    'kernel_sz': 5,\n",
    "    'hidden_dim': 128,\n",
    "    'n_hidden_layer': 4,\n",
    "    'dropout_rate': 0.1,\n",
    "    'display_step': 10,\n",
    "    'generate_step': 100,\n",
    "    'beam_size': 5,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_text(file_path):\n",
    "    with open(file_path) as f:\n",
    "        text = f.read()\n",
    "    \n",
    "    char2idx = {c: i+3 for i, c in enumerate(set(text))}\n",
    "    char2idx['<pad>'] = 0\n",
    "    char2idx['<start>'] = 1\n",
    "    char2idx['<end>'] = 2\n",
    "    \n",
    "    ints = np.array([char2idx[char] for char in list(text)])\n",
    "    return ints, char2idx\n",
    "\n",
    "def next_batch(ints):\n",
    "    len_win = params['seq_len'] * params['batch_size']\n",
    "    for i in range(0, len(ints)-len_win, params['text_iter_step']):\n",
    "        clip = ints[i: i+len_win]\n",
    "        yield clip.reshape([params['batch_size'], params['seq_len']])\n",
    "        \n",
    "def input_fn(ints):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: next_batch(ints), tf.int32, tf.TensorShape([None, params['seq_len']]))\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def start_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<start>'])\n",
    "    return tf.concat([_x, x], 1)\n",
    "\n",
    "def end_sent(x):\n",
    "    _x = tf.fill([tf.shape(x)[0], 1], params['char2idx']['<end>'])\n",
    "    return tf.concat([x, _x], 1)\n",
    "\n",
    "def embed_seq(x, vocab_sz, embed_dim, name, zero_pad=True):\n",
    "    embedding = tf.get_variable(name, [vocab_sz, embed_dim])\n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0)\n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def position_encoding(inputs):\n",
    "    repr_dim = inputs.get_shape()[-1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(tf.shape(inputs)[1]), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])\n",
    "\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "\n",
    "def cnn_block(x, dilation_rate, pad_sz, is_training):\n",
    "    x = layer_norm(x)\n",
    "    x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, params['hidden_dim']])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = params['hidden_dim'],\n",
    "                          kernel_size = params['kernel_sz'],\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def forward(inputs, reuse, is_training):\n",
    "    with tf.variable_scope('model', reuse=reuse):\n",
    "        x = embed_seq(inputs, params['vocab_size'], params['hidden_dim'], 'word_embedding')\n",
    "        x += position_encoding(x)\n",
    "        \n",
    "        for i in range(params['n_hidden_layer']):\n",
    "            dilation_rate = 2 ** i\n",
    "            pad_sz = (params['kernel_sz'] - 1) * dilation_rate\n",
    "            with tf.variable_scope('block_%d'%i, reuse=reuse):\n",
    "                x += cnn_block(x, dilation_rate, pad_sz, is_training)\n",
    "        \n",
    "        logits = tf.layers.dense(x, params['vocab_size'])\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beam_search_decoding():\n",
    "    batch_size = 1\n",
    "    initial_ids = tf.constant(params['char2idx']['<start>'], tf.int32, [batch_size])\n",
    "    \n",
    "    def symbols_to_logits(ids):\n",
    "        logits = forward(ids, reuse=True, is_training=False)\n",
    "        return logits[:, tf.shape(ids)[1]-1, :]\n",
    "    \n",
    "    final_ids, final_probs = beam_search.beam_search(\n",
    "        symbols_to_logits,\n",
    "        initial_ids,\n",
    "        params['beam_size'],\n",
    "        params['seq_len'],\n",
    "        params['vocab_size'],\n",
    "        0.0,\n",
    "        eos_id = params['char2idx']['<end>'])\n",
    "    \n",
    "    return final_ids[0, 0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 86\n"
     ]
    }
   ],
   "source": [
    "ints, params['char2idx'] = parse_text('../temp/anna.txt')\n",
    "params['vocab_size'] = len(params['char2idx'])\n",
    "params['idx2char'] = {i: c for c, i in params['char2idx'].items()}\n",
    "print('Vocabulary size:', params['vocab_size'])\n",
    "\n",
    "X = input_fn(ints)\n",
    "logits = forward(start_sent(X), reuse=False, is_training=True)\n",
    "\n",
    "ops = {}\n",
    "ops['global_step'] = tf.Variable(0, trainable=False)\n",
    "\n",
    "targets = end_sent(X)\n",
    "ops['loss'] = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(\n",
    "    logits = logits,\n",
    "    targets = targets,\n",
    "    weights = tf.to_float(tf.ones_like(targets))))\n",
    "\n",
    "ops['train'] = tf.train.AdamOptimizer().minimize(ops['loss'], global_step=ops['global_step'])\n",
    "\n",
    "ops['generate'] = beam_search_decoding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1 | Loss 7.378\n",
      "Step 10 | Loss 3.380\n",
      "Step 20 | Loss 3.177\n",
      "Step 30 | Loss 3.090\n",
      "Step 40 | Loss 3.022\n",
      "Step 50 | Loss 2.962\n",
      "Step 60 | Loss 2.899\n",
      "Step 70 | Loss 2.837\n",
      "Step 80 | Loss 2.753\n",
      "Step 90 | Loss 2.682\n",
      "Step 100 | Loss 2.607\n",
      "\n",
      "<start> he the the the the the the the the ther the the the the thes the the the the the thin the the the the ther the the the thes the the the thin the the the the the the the ther the the the thes the the<end>\n",
      "\n",
      "Step 110 | Loss 2.557\n",
      "Step 120 | Loss 2.505\n",
      "Step 130 | Loss 2.457\n",
      "Step 140 | Loss 2.417\n",
      "Step 150 | Loss 2.384\n",
      "Step 160 | Loss 2.348\n",
      "Step 170 | Loss 2.327\n",
      "Step 180 | Loss 2.297\n",
      "Step 190 | Loss 2.268\n",
      "Step 200 | Loss 2.240\n",
      "\n",
      "<start> and her and and her and and her and and her and he ther and an thing he thing of he ther and on the thing he thing he thing on the sher and he thing of the she ther and and he ther and her and he the\n",
      "\n",
      "Step 210 | Loss 2.210\n",
      "Step 220 | Loss 2.188\n",
      "Step 230 | Loss 2.166\n",
      "Step 240 | Loss 2.148\n",
      "Step 250 | Loss 2.121\n",
      "Step 260 | Loss 2.097\n",
      "Step 270 | Loss 2.070\n",
      "Step 280 | Loss 2.046\n",
      "Step 290 | Loss 2.020\n",
      "Step 300 | Loss 1.992\n",
      "\n",
      "<start> the wall the wist and wath the wather of the could of the could of the ware the could and an the could of the her and, and an the her and, and her and, and an the could, and of the could, and and he \n",
      "\n",
      "Step 310 | Loss 1.964\n",
      "Step 320 | Loss 1.932\n",
      "Step 330 | Loss 1.907\n",
      "Step 340 | Loss 1.881\n",
      "Step 350 | Loss 1.846\n",
      "Step 360 | Loss 1.811\n",
      "Step 370 | Loss 1.780\n",
      "Step 380 | Loss 1.744\n",
      "Step 390 | Loss 1.710\n",
      "Step 400 | Loss 1.665\n",
      "\n",
      "<start> the could of the could of the child of the could on the child of the could of the child on the could of the child of the child on the could of the child on the could of the child on the could of the \n",
      "\n",
      "Step 410 | Loss 1.628\n",
      "Step 420 | Loss 1.591\n",
      "Step 430 | Loss 1.537\n",
      "Step 440 | Loss 1.501\n",
      "Step 450 | Loss 1.461\n",
      "Step 460 | Loss 1.417\n",
      "Step 470 | Loss 1.366\n",
      "Step 480 | Loss 1.328\n",
      "Step 490 | Loss 1.285\n",
      "Step 500 | Loss 1.236\n",
      "\n",
      "<start> the children the children's thing out an the of the childron.\n",
      "\n",
      "\n",
      "Stepan Arkadyevitch could not ge of his canding with he waid him with the must to her and of the room.\" He thout of the children, and w\n",
      "\n",
      "Step 510 | Loss 1.191\n",
      "Step 520 | Loss 1.146\n",
      "Step 530 | Loss 1.107\n",
      "Step 540 | Loss 1.066\n",
      "Step 550 | Loss 1.022\n",
      "Step 560 | Loss 0.980\n",
      "Step 570 | Loss 0.949\n",
      "Step 580 | Loss 0.910\n",
      "Step 590 | Loss 0.883\n",
      "Step 600 | Loss 0.850\n",
      "\n",
      "<start> the children the could not thing was a wald to him, and her face of the said.\n",
      "\n",
      "\"We shouped in the maid he said to She broghed. And that he was in the but her, and that he was and of the saying of the\n",
      "\n",
      "Step 610 | Loss 0.823\n",
      "Step 620 | Loss 0.786\n",
      "Step 630 | Loss 0.763\n",
      "Step 640 | Loss 0.736\n",
      "Step 650 | Loss 0.721\n",
      "Step 660 | Loss 0.690\n",
      "Step 670 | Loss 0.664\n",
      "Step 680 | Loss 0.645\n",
      "Step 690 | Loss 0.636\n",
      "Step 700 | Loss 0.614\n",
      "\n",
      "<start> the children the door oft that any sparing at the when he coused her han come in the beres, and agraly and her face ly and her and what the must he couldry of the roped to thoured of her male of the \n",
      "\n",
      "Step 710 | Loss 0.612\n",
      "Step 720 | Loss 0.594\n",
      "Step 730 | Loss 0.588\n",
      "Step 740 | Loss 0.579\n",
      "Step 750 | Loss 0.566\n",
      "Step 760 | Loss 0.540\n",
      "Step 770 | Loss 0.539\n",
      "Step 780 | Loss 0.531\n",
      "Step 790 | Loss 0.520\n",
      "Step 800 | Loss 0.507\n",
      "\n",
      "<start> the children the doore of the mon; of ofe the the comest of ons of the board, and dy them his ould to you mest into his all, an the liferatiom ot kession to the malugible very with the roust of the c\n",
      "\n",
      "Step 810 | Loss 0.509\n",
      "Step 820 | Loss 0.498\n",
      "Step 830 | Loss 0.497\n",
      "Step 840 | Loss 0.484\n",
      "Step 850 | Loss 0.479\n",
      "Step 860 | Loss 0.465\n",
      "Step 870 | Loss 0.458\n",
      "Step 880 | Loss 0.456\n",
      "Step 890 | Loss 0.446\n",
      "Step 900 | Loss 0.448\n",
      "\n",
      "<start> the couldren, enter, in would do a love your?\" shisusting of him in bout of the ramint the door.\n",
      "\n",
      "Stepan Arkadyevitch comed the ghe weppecting, and to Oblonsky.\n",
      "Stepan Arkadyevitch, as onter, she we<end>\n",
      "\n",
      "Step 910 | Loss 0.443\n",
      "Step 920 | Loss 0.430\n",
      "Step 930 | Loss 0.440\n",
      "Step 940 | Loss 0.427\n",
      "Step 950 | Loss 0.431\n",
      "Step 960 | Loss 0.420\n",
      "Step 970 | Loss 0.419\n",
      "Step 980 | Loss 0.412\n",
      "Step 990 | Loss 0.402\n",
      "Step 1000 | Loss 0.398\n",
      "\n",
      "<start> the for to that\n",
      "her in that Levin with the seed him shen the carvic ard well as the chmer, and the children, and for that rearacmed in the sarrick? Levin the would not. He the town of the stofe of th\n",
      "\n",
      "Step 1010 | Loss 0.400\n",
      "Step 1020 | Loss 0.398\n",
      "Step 1030 | Loss 0.392\n",
      "Step 1040 | Loss 0.389\n",
      "Step 1050 | Loss 0.397\n",
      "Step 1060 | Loss 0.394\n",
      "Step 1070 | Loss 0.407\n",
      "Step 1080 | Loss 0.392\n",
      "Step 1090 | Loss 0.386\n",
      "Step 1100 | Loss 0.388\n",
      "\n",
      "<start> the said to Matvey wher he came in.\n",
      "\n",
      "\"Yes, sir.\"\n",
      "\n",
      "Stepan Arkadyevitch put on his fur coat and went out onto the steps.\n",
      "\n",
      "\"You won't dine at home?\" said Matvey, seeing him off.\n",
      "\n",
      "\"That's as it happens. \n",
      "\n",
      "Step 1110 | Loss 0.379\n",
      "Step 1120 | Loss 0.373\n",
      "Step 1130 | Loss 0.376\n",
      "Step 1140 | Loss 0.374\n",
      "Step 1150 | Loss 0.364\n",
      "Step 1160 | Loss 0.369\n",
      "Step 1170 | Loss 0.381\n",
      "Step 1180 | Loss 0.366\n",
      "Step 1190 | Loss 0.369\n",
      "Step 1200 | Loss 0.361\n",
      "\n",
      "<start> the district councils, or ever could be,\" he\n",
      "began, as though some one had just insulted him. \"On one side it's a\n",
      "plaything; they play at being a parliament, and I'm neither young enough\n",
      "nor old enou\n",
      "\n",
      "Step 1210 | Loss 0.372\n",
      "Step 1220 | Loss 0.359\n",
      "Step 1230 | Loss 0.364\n",
      "Step 1240 | Loss 0.356\n",
      "Step 1250 | Loss 0.365\n",
      "Step 1260 | Loss 0.366\n",
      "Step 1270 | Loss 0.366\n",
      "Step 1280 | Loss 0.362\n",
      "Step 1290 | Loss 0.359\n",
      "Step 1300 | Loss 0.355\n",
      "\n",
      "<start> he had\n",
      "been tolk that\n",
      "he would not get a position with the salary he required, especially as\n",
      "he expected nothing out of the way; he only wanted what the men of his\n",
      "own age and standing did get, and h\n",
      "\n",
      "Step 1310 | Loss 0.354\n",
      "Step 1320 | Loss 0.354\n",
      "Step 1330 | Loss 0.353\n",
      "Step 1340 | Loss 0.361\n",
      "Step 1350 | Loss 0.351\n",
      "Step 1360 | Loss 0.355\n",
      "Step 1370 | Loss 0.351\n",
      "Step 1380 | Loss 0.357\n",
      "Step 1390 | Loss 0.339\n",
      "Step 1400 | Loss 0.334\n",
      "\n",
      "<start>the consequently alw of them the matter. Levin Steple.\n",
      "\n",
      "\"What's consicters that Levin tare, seever that they Stepan Arkadyevitch.\n",
      "\n",
      "\"Wo have succeeded in getting the information from the government\n",
      "dep\n",
      "\n",
      "Step 1410 | Loss 0.344\n",
      "Step 1420 | Loss 0.340\n",
      "Step 1430 | Loss 0.333\n",
      "Step 1440 | Loss 0.333\n",
      "Step 1450 | Loss 0.335\n",
      "Step 1460 | Loss 0.321\n",
      "Step 1470 | Loss 0.330\n",
      "Step 1480 | Loss 0.325\n",
      "Step 1490 | Loss 0.334\n",
      "Step 1500 | Loss 0.339\n",
      "\n",
      "<start> that they was not alone of the same Levin in the dowas forr of the professor, he noticed\n",
      "that they connected these scientific questions with those spiritual\n",
      "problems, that at times they almost toun w\n",
      "\n",
      "Step 1510 | Loss 0.330\n",
      "Step 1520 | Loss 0.326\n",
      "Step 1530 | Loss 0.322\n",
      "Step 1540 | Loss 0.321\n",
      "Step 1550 | Loss 0.325\n",
      "Step 1560 | Loss 0.330\n",
      "Step 1570 | Loss 0.324\n",
      "Step 1580 | Loss 0.317\n",
      "Step 1590 | Loss 0.321\n",
      "Step 1600 | Loss 0.324\n",
      "\n",
      "<start> that they wase that they were close of the question in pleas you, and wenceall of his have a grea bug the conse--ato his the mart to the professor of marry with that here.\n",
      "\n",
      "Levin facised, and the uf<end>\n",
      "\n",
      "Step 1610 | Loss 0.320\n",
      "Step 1620 | Loss 0.316\n",
      "Step 1630 | Loss 0.315\n",
      "Step 1640 | Loss 0.322\n",
      "Step 1650 | Loss 0.328\n",
      "Step 1660 | Loss 0.316\n",
      "Step 1670 | Loss 0.324\n",
      "Step 1680 | Loss 0.317\n",
      "Step 1690 | Loss 0.318\n",
      "Step 1700 | Loss 0.308\n",
      "\n",
      "<start>ing that he had for the could not be ate with all there was a starting his vetcres.\n",
      "\n",
      "\"That stare with the Levins and the Shtcherbatskys were old, noble Moscow\n",
      "families, and had always been on intimate\n",
      "\n",
      "Step 1710 | Loss 0.315\n",
      "Step 1720 | Loss 0.330\n",
      "Step 1730 | Loss 0.336\n",
      "Step 1740 | Loss 0.329\n",
      "Step 1750 | Loss 0.327\n",
      "Step 1760 | Loss 0.323\n",
      "Step 1770 | Loss 0.325\n",
      "Step 1780 | Loss 0.323\n",
      "Step 1790 | Loss 0.320\n",
      "Step 1800 | Loss 0.325\n",
      "\n",
      "<start>the conversation.\n",
      "\n",
      "A little man in spectacles, with a narrow forehead, tore himself from\n",
      "the discussion for an instant to greet Levin, and then went on talking\n",
      "without paying any further attention to \n",
      "\n",
      "Step 1810 | Loss 0.320\n",
      "Step 1820 | Loss 0.319\n",
      "Step 1830 | Loss 0.314\n",
      "Step 1840 | Loss 0.317\n",
      "Step 1850 | Loss 0.310\n",
      "Step 1860 | Loss 0.315\n",
      "Step 1870 | Loss 0.315\n",
      "Step 1880 | Loss 0.327\n",
      "Step 1890 | Loss 0.319\n",
      "Step 1900 | Loss 0.316\n",
      "\n",
      "<start>the conversation.\n",
      "\n",
      "A little man in spectacles,\" sa<end><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1910 | Loss 0.318\n",
      "Step 1920 | Loss 0.313\n",
      "Step 1930 | Loss 0.316\n",
      "Step 1940 | Loss 0.323\n",
      "Step 1950 | Loss 0.313\n",
      "Step 1960 | Loss 0.312\n",
      "Step 1970 | Loss 0.309\n",
      "Step 1980 | Loss 0.309\n",
      "Step 1990 | Loss 0.305\n",
      "Step 2000 | Loss 0.298\n",
      "\n",
      "<start> that he had no sort of proof that he would be rejected. And\n",
      "he had now come to Moscow with a firm determination to make an offer,\n",
      "and get married if he were accepted. Or ... he could not conceive wha\n",
      "\n",
      "Step 2010 | Loss 0.300\n",
      "Step 2020 | Loss 0.306\n",
      "Step 2030 | Loss 0.311\n",
      "Step 2040 | Loss 0.313\n",
      "Step 2050 | Loss 0.301\n",
      "Step 2060 | Loss 0.301\n",
      "Step 2070 | Loss 0.305\n",
      "Step 2080 | Loss 0.309\n",
      "Step 2090 | Loss 0.304\n",
      "Step 2100 | Loss 0.297\n",
      "\n",
      "<start> that he had no sort of proof that he would be rejected. And\n",
      "he had now come to Moscow with a firm determination to make an offer,\n",
      "and get married if he were accepted. Or ... he could not conceive wha\n",
      "\n",
      "Step 2110 | Loss 0.310\n",
      "Step 2120 | Loss 0.302\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "while True:\n",
    "    try:\n",
    "        _, step, loss = sess.run([ops['train'], ops['global_step'], ops['loss']])\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        break\n",
    "    else:\n",
    "        if step % params['display_step'] == 0 or step == 1:\n",
    "            print(\"Step %d | Loss %.3f\" % (step, loss))\n",
    "        if step % params['generate_step'] == 0 and step > 1:\n",
    "            ints = sess.run(ops['generate'])\n",
    "            print('\\n'+''.join([params['idx2char'][i] for i in ints])+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
